{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二周作业 在Rental Listing Inquiries数据上练习分类方法\n",
    "Rental Listing Inquiries数据集是Kaggle平台上的一个分类竞赛任务，需要根据公寓的特征来预测其受欢迎程度（用户感兴趣程度分为高、中、低三类）。其中房屋的特征x共有14维，响应值y为用户对该公寓的感兴趣程度。评价标准为logloss。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "导入必要的包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import log_loss  \n",
    "from matplotlib import pyplot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取CSV数据（特征工程生成的CSV文件）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 221 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 221 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath + \"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 221 entries, bathrooms to interest_level\n",
      "dtypes: float64(6), int64(215)\n",
      "memory usage: 83.2 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>listing_id</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7142618</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7210040</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7103890</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7143442</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6860601</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 221 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   listing_id  bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  \\\n",
       "0     7142618        1.0         1   2950      1475.000000     1475.000000   \n",
       "1     7210040        1.0         2   2850      1425.000000      950.000000   \n",
       "2     7103890        1.0         1   3758      1879.000000     1879.000000   \n",
       "3     7143442        1.0         2   3300      1650.000000     1100.000000   \n",
       "4     6860601        2.0         2   4900      1633.333333     1633.333333   \n",
       "\n",
       "   room_diff  room_num  Year  Month  ...   virtual  walk  walls  war  washer  \\\n",
       "0        0.0       2.0  2016      6  ...         0     0      0    0       0   \n",
       "1       -1.0       3.0  2016      6  ...         0     0      0    1       0   \n",
       "2        0.0       2.0  2016      6  ...         0     0      0    0       0   \n",
       "3       -1.0       3.0  2016      6  ...         0     0      0    0       0   \n",
       "4        0.0       4.0  2016      4  ...         0     0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  \n",
       "0      0           0     0        0     0  \n",
       "1      0           0     0        0     0  \n",
       "2      0           0     0        0     0  \n",
       "3      0           1     0        0     0  \n",
       "4      0           0     0        0     0  \n",
       "\n",
       "[5 rows x 221 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(dpath + \"RentListingInquries_FE_test.csv\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 74659 entries, 0 to 74658\n",
      "Columns: 221 entries, listing_id to work\n",
      "dtypes: float64(6), int64(215)\n",
      "memory usage: 125.9 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['interest_level']   \n",
    "X_train = train.drop([\"interest_level\"], axis=1)\n",
    "test_Id = test['listing_id']\n",
    "X_test = test.drop([\"listing_id\"], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据标准化处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "ss_X = StandardScaler()\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "X_test = ss_X.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "默认逻辑回归模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is: [ 0.6857808   0.68499999  0.68042685  0.66931504  0.6825717 ]\n",
      "cv logloss is: 0.680618876971\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print('logloss of each fold is:',-loss)\n",
    "print('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "正则化得逻辑回归和参数调优\n",
    "目标函数为：J = sum(logloss(f(xi), yi)) + （1/C）* penalty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train, y_train, train_size = 0.33,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "def fit_grid_point_LR(penalty, C, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集训练LR\n",
    "    LR = LogisticRegression(penalty=penalty, C=C)\n",
    "    LR.fit(X_train, y_train)\n",
    "    \n",
    "    # 在训练集和校验集上测试\n",
    "    y_train_pred = LR.predict_proba(X_train)\n",
    "    y_val_pred = LR.predict_proba(X_val)\n",
    "    logloss_val = log_loss(y_val,y_val_pred)\n",
    "    logloss_train = log_loss(y_train, y_train_pred)\n",
    "    \n",
    "    print(\"logloss on test: %f and on train: %f with C = %f and penalty = %s\"%(logloss_val, logloss_train, C, penalty) )\n",
    "    return logloss_val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "调用调参方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss on test: 0.810482 and on train: 0.810799 with C = 0.001000 and penalty = l1\n",
      "logloss on test: 0.774785 and on train: 0.767327 with C = 0.001000 and penalty = l2\n",
      "logloss on test: 0.734726 and on train: 0.733116 with C = 0.010000 and penalty = l1\n",
      "logloss on test: 0.728447 and on train: 0.706835 with C = 0.010000 and penalty = l2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss on test: 0.687446 and on train: 0.668496 with C = 0.100000 and penalty = l1\n",
      "logloss on test: 0.717933 and on train: 0.674018 with C = 0.100000 and penalty = l2\n",
      "logloss on test: 0.709741 and on train: 0.654235 with C = 1.000000 and penalty = l1\n",
      "logloss on test: 0.733107 and on train: 0.653050 with C = 1.000000 and penalty = l2\n",
      "logloss on test: 0.747421 and on train: 0.651511 with C = 10.000000 and penalty = l1\n",
      "logloss on test: 0.756848 and on train: 0.648259 with C = 10.000000 and penalty = l2\n",
      "logloss on test: 0.765701 and on train: 0.651188 with C = 100.000000 and penalty = l1\n",
      "logloss on test: 0.770318 and on train: 0.646984 with C = 100.000000 and penalty = l2\n",
      "logloss on test: 0.769027 and on train: 0.651018 with C = 1000.000000 and penalty = l1\n",
      "logloss on test: 0.780200 and on train: 0.646302 with C = 1000.000000 and penalty = l2\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "\n",
    "logloss_s = []\n",
    "for i, OneC in enumerate(Cs):\n",
    "    for j, onePenalty in enumerate(penaltys):\n",
    "        tmp = fit_grid_point_LR(onePenalty, OneC, X_train_part, y_train_part, X_val, y_val)\n",
    "        logloss_s.append(tmp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "显示个正则参数效果图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XlYlGX3wPHvAVHEDfcNBBdcU0xR\nxK1MzSXLLMvMtXwrS3vb932x/de+uZWvZtqm5praqiYu4C7u5oK7uKKCwNy/P56hkJAZYIYZhvO5\nLq6Ymed55owph/vc93NuMcaglFJK5cbP0wEopZTyfposlFJKOaTJQimllEOaLJRSSjmkyUIppZRD\nmiyUUko5pMlCKaWUQ5oslFJKOaTJQimllEMlPB2Aq1SpUsWEh4d7OgyllCpS4uPjjxtjqjo6zmeS\nRXh4OHFxcZ4OQymlihQR2evMcVqGUkop5ZAmC6WUUg5pslBKKeWQz8xZKKWKt7S0NBITE0lJSfF0\nKF4pMDCQkJAQAgIC8nW+JgullE9ITEykXLlyhIeHIyKeDserGGNISkoiMTGRunXr5usaWoZSSvmE\nlJQUKleurIkiByJC5cqVCzTq0mShlPIZmigur6B/NsU+WZw+n8a7i7ez48hZT4eilFJeq9gnC5sx\nfP7HLv4Xu8fToSilfER0dDQtW7akTp06VK1alZYtW9KyZUv27NmTp+vMmDGDrVu35vhaeno6wcHB\nANhsNnr06EFwcDA33nhjQcPPUbGf4K5YpiTXt6jFzDUHeKJnY8oF5m+lgFJKZVq5ciUAkyZNIi4u\njo8//jhf15kxYwZ+fn40btw41+NEhMcff5yzZ88yadKkfL2XI8V+ZAEwNCaMcxczmLn2gKdDUUr5\nuAULFhATE0OrVq0YMGAA586dA+Cxxx6jadOmtGjRgieeeIKlS5cyf/58HnroIYejEhGha9eulC1b\n1m1xF/uRBUBkaDCRIRWYHLuXIe3CdJJMqSLupTmbSTh4xqXXbFqrPC9c36xA1zh69ChvvPEGv/zy\nC0FBQYwZM4YPPviAESNGMH/+fDZv3oyIcOrUKYKDg+nduzf9+/d3W2kpL3RkYTe4XRg7jyYTuzvJ\n06EopXzU8uXLSUhIoH379rRs2ZKpU6eyZ88eKlWqhJ+fH3fddRczZ86kTJkyng71X3RkYXd9ZC3G\nzN/CVyv20r5+FU+Ho5QqgIKOANzFGEPPnj2ZMmXKv16Li4tj8eLFTJ8+nc8++4xFixZd8vqePXv+\nHmGMHj2a4cOHF0bIf9NkYRcY4M+AqFAmLPuLw6dTqFEh0NMhKaV8TPv27XnggQfYvXs39erV49y5\ncxw8eJAaNWqQkpJCnz59iI6OpmnTpgCUK1eOs2etZf3h4eGsW7fu72ulp6cXauxahspiUHQYNmP4\netU+T4eilPJB1atXZ+LEiQwYMIDIyEjat2/P9u3bOX36NNdddx2RkZFcc801vPvuuwAMHDiQ1157\nzalltzExMQwcOJCFCxcSEhLCL7/84tLYxRjj0gt6SlRUlHHF5kd3fLmKTQfP8OcT11CyhOZSpYqK\nLVu20KRJE0+H4dVy+jMSkXhjTJSjc/WnYTZDY8I5djaVRQmHPR2KUkp5DU0W2VzVsCqhlUozOdap\nnQaVUqpY0GSRjZ+fMDg6jFV/nWDrYdeu01ZKqaJKk0UObo0KpVQJP75aoaMLpZQCNycLEekpIttE\nZKeIPJnD63VE5DcRWSsiG0Skd5bXnrKft01EergzzuwqlinJ9ZFWv6izKWmF+dZKKeWV3JYsRMQf\n+AToBTQFBopI02yHPQt8a4y5ErgN+NR+blP742ZAT+BT+/UKzZB2Vr+oGWu0X5RSSrlzZNEW2GmM\n2W2MuQhMB/pmO8YA5e3fVwAO2r/vC0w3xqQaY/4CdtqvV2gy+0VNWbEXX1lerJQqHIXdojw+Pp52\n7dpxxRVX0KJFC77//vuCfoR/cecd3LWB/VkeJwLR2Y55EVgkIvcDZYBuWc5dke3c2u4J8/KGxITz\n6Hfrid2dpC1AlFJOK+wW5WXLlmXq1KnUr1+fxMREoqKi6NGjB+XKlcvX++bEnSOLnFq3Zv8VfSAw\nyRgTAvQGpoiIn5PnIiJ3i0iciMQdO3aswAFn16dFTYKDApiiy2iVUi7ijhbljRo1on79+gCEhIRQ\nuXJljh8/7tK43TmySARCszwO4Z8yU6YRWHMSGGNiRSQQqOLkuRhjxgHjwLqD22WR22XtF3Xo9AVq\nVijt6rdQSrnDgifh8EbXXrNGc+j1RoEuURgtypcvXw5YvaRcyZ0ji9VAhIjUFZGSWBPWs7Mdsw/o\nCiAiTYBA4Jj9uNtEpJSI1AUigFVujPWyBrez+kVNW7Xf8cFKKZULd7coP3DgAMOHD2fSpEku35fH\nbSMLY0y6iIwGFgL+wBfGmM0i8jIQZ4yZDTwCjBeRh7DKTMONNZu8WUS+BRKAdGCUMSbDXbHmJrRS\nEF0aVWPaqn2M7tJA+0UpVRQUcATgLu5sUZ7ZjPDNN9+kTZs2Lo/drS3KjTHzgfnZnns+y/cJQIfL\nnDsGGOPO+Jw1pF0Yd0xazcLNh7k+spanw1FKFVHualGemppK3759GTFiBP369XNL7PprshOualiV\nOpWCmKJ3dCulCsBdLcqnTZvG8uXLmThx4t/LdDdudO2cjbYod9K4Jbt4bf5WfnqwE41rlHd8glKq\nUGmLcse0RXkhuKW11S9Kl9EqpYojTRZO+rtf1FrtF6WUKn40WeTB0Jgwzmu/KKW8lq+U1d2hoH82\nmizyoEWI9otSylsFBgaSlJSk/zZzYIwhKSmJwMDAfF/DrUtnfdHf/aJ2JdG+gfaLUspbhISEkJiY\niDta//iCwMBAQkJC8n2+Jos86tOiJmPmJTBlxV5NFkp5kYCAAOrWrevpMHyWlqHyKDDAn1vbhLIo\n4QiHTl/wdDhKKVUoNFnkw+Boe7+olfs8HYpSShUKTRb58He/qNX7uZhu83Q4Sinldpos8mlITBjH\nzqaycPNhT4eilFJup8kin66KsPeL0ju6lVLFgCaLfPLzEwa3q8OqPSfYeviMp8NRSim30mRRANov\nSilVXGiyKICs/aLOaL8opZQP02RRQJn9omZqvyillA/TZFFALUKCiQwN1n5RSimfpsnCBYa0C2Pn\n0WRidyV5OhSllHILTRYu0KdFTSoGBTBZJ7qVUj5Kk4ULZPaLWrxF+0UppXyTJgsX0X5RSilfpsnC\nRTL7RX29SvtFKaV8jyYLFxoSE8bxZO0XpZTyPW5NFiLSU0S2ichOEXkyh9ffE5F19q/tInIqy2tv\nichmEdkiIh+KiLgzVlfQflFKKV/ltmQhIv7AJ0AvoCkwUESaZj3GGPOQMaalMaYl8BEww35ue6AD\n0AK4AmgDXOWuWF1F+0UppTyiEO7xcufIoi2w0xiz2xhzEZgO9M3l+IHANPv3BggESgKlgADgiBtj\ndZlbo7RflFKqEJw5CKvGw+Qb4duhbn87d+7BXRvYn+VxIhCd04EiEgbUBX4FMMbEishvwCFAgI+N\nMVtyOO9u4G6AOnXquDT4/AoOKskN9n5RT/RqTPnAAE+HpJTyBcbA8e2wdS5snQcH4q3nKzeAK/q7\n/e3dmSxymmO43FjpNuB7Y0wGgIg0AJoAIfbXF4tIZ2PMkksuZsw4YBxAVFSU1/TaGBITxnfxicyI\nT2R4B91AXimVTzYbHIj7J0Ek7bSer90auj4PjftA1UaFEoo7k0UiEJrlcQhw8DLH3gaMyvK4H7DC\nGJMMICILgHbAkhzO9TpZ+0UNax9OEZibV0p5i/RU+GuJlSC2LYDkI+BXAup2hnb3QqPeUL5WoYfl\nzmSxGogQkbrAAayEcHv2g0SkEVARiM3y9D7gLhF5HWuEchXwvhtjdbmh7cJ45Lv1xO5Kon2DKp4O\nRynlzVJOw47F1uhhx2K4eBZKloWI7tbooUE3KB3s0RDdliyMMekiMhpYCPgDXxhjNovIy0CcMWa2\n/dCBwHRzacvW74FrgI1YpaufjDFz3BWrO1zXoiavzktgcuxeTRZKqX87cwi2zbcSxF9LwJYGZarC\nFTdZCaJuZwgI9HSUf3PnyAJjzHxgfrbnns/2+MUczssA7nFnbO6W2S9qwtK/OHT6AjUrlPZ0SEop\nTzu+wyovbZlrzUUAVKpnlZca94GQKPDz92yMl+HWZFHcDY4OY9yS3UxbuY+Hry2cSSillBex2eDg\nmn8mqI9vt56vdSVc8yw0vt6aoC4C85qaLAAOroNqTaFESZdeNrRSENfY+0WNviaCkiW0u4pSPi/9\nIuxZYiWHrfMh+bA1QR3eEdreDY16QYUQx9fxMposju+A8V2g0yNWpnexwTFh/PLlan7afJgbIgt/\nBYNSqhCknIGdP9snqBdB6hkIKAMR3azyUkR3KF3R01EWiCaLKhHQYgAsfRca9oKQ1i69/FURVQmr\nHMRXsXs1WSjlS84eyTJB/QdkXISgKtC0r5Ug6l3tVRPUBaXJAqDnG9ZqhJn3wMilEOC6yWg/P2Fw\ndBhj5m9h6+EzNK5R3mXXVkoVsqRdsGWOlSASVwMGKoZb5aXGfSC0rddOUBeUJguw1i/3/Rim9INf\nXoaer7v08rdEhfDOom1Mjt3La/2au/TaSik3stng0Fr7/MM8OLbVer5mJHR5BhpfB9WaFIkJ6oLS\nZJGp/jXQ5j+w4lPrDsm6nVx26cx+UbPWHuBJ7RellHdLvwh7l/0zQX32IIg/hHeAqDutnw/BoY6v\n42M0WWTV/WXY+Qv8eB/cuxxKlXPZpYfGhGu/KKW8VerZfyaoty+C1NMQEAQNukLjFyDiWgiq5Oko\nPUqTRVYly0C/z+GLnrDwGbjhQ5ddunlIBVpqvyilvEfyUav30ta5sPt3a4K6dCVocj00yZyg1ptp\nM2myyK5OO2h/Pyz/0PpLE9HdZZceYu8XtXxXEh20BYhShc9mg92/wspx1hJXDATXgTZ3WfMPodHg\nrz8Wc6J/Kjnp8ozVzOvH0XBfrMuGn9e1qMmY+VuYErtXk4VShSnlDKyfBqvGWW2+y1Sz7q1q1g+q\nNysWE9QFpckiJwGB0O8zmNANFjwON09wyWUDA/y5NSqU8Ut3a78opQrD8Z1Wglj3tdXJtXYU3DQe\nmt7o8o4N7pKWYePo2VQOn07hyJmUf/6b5fvwKmWYdEdbt8ahyeJyal0JnR+D31+31k83u9Ellx0U\nXYexS3bx9cp9PKL9opRyPZsNdi6GlWNh1y/gFwBX3AzRd1ubBnkJYwxnUtL/TgCHz6RwJPO/fyeD\nVJLOpf5ri+2SJfyoUT6QGuUDuaJ2BZrVquD2eDVZ5KbTI9YE2NyHIKw9lK1W4Etm9ouatmo/92u/\nKKVcJ+U0rJ0Kq8fDid1QtoZVUm493CX/dvMi+2gg84f/P8nAeu1CWsa/zq1UpiTVywdSo3wpmteu\nYP8+kOoVAv9OEMFBAYW+SEaTRW78A6DfWBjbGeY8ALd97ZLa5hDtF6WU6xzdapWa1k+HtHPWJPU1\nz0KTG6x/wy5UoNGAvx/VK5SiRvlAmtUqT9fG1ahRIdBKBvZEUK18KUqV8M47wDVZOFKtMXR9DhY9\na02QtfzXZn951tneL2pK7B5NFkrlhy0Dti+EVWOtZa/+paB5f6vtRq2W+bpkWoaNY2dTL0kAzo4G\nKgYF/P1D/4paFS5JAJnfV/TAaMCVNFk4o9191p2cC56wdq8qYHvhrP2ithw6Q5Oa2i9KKadcOAlr\npsDqCXBqL5SvDV2fh1bDoIzzKwzX7DvJzDUHOJRlsvh4snOjgerZSkLVypciMMA7RwOuJCb7n04R\nFRUVZeLi4tz3Bid2w2cdIbQNDJ4JfgWbazh1/iLRr/3Cza1DtF+UUo4cSbBGERu+hbTzENbhn+Z9\nebgv4vDpFN78aSsz1x6gbKkShFQs/a8RgC+NBpwhIvHGmChHx+nIwlmV6sG1r8C8hyFuIrS9q0CX\n035RSjmQkW61AF81DvYshRKB0PwWiL4HauTtF6yUtAwmLvuLT37bSbrNMKpLfe67ugFlSumPQGfp\nn1ReRN1ptQZY/LzVeLBy/QJdLrNf1A/xidyh/aKUspw/AWv+B6snwun9UCEUur0ErYbm+QZZYwwL\nNx/m1XlbSDx5gZ7NavB07ybUqRzkpuB9lyaLvBCBGz6GT2Ng1n1wx/wC9a7P2i9quPaLUsXdoQ1W\nqWnj95CeAuGdrO0CGvbKVwuOLYfO8PKcBGJ3J9Goejmm/idaOycUgCaLvKpQG3q/ZW2UFPsxdHig\nQJcbGhPGw99qvyhVTGWkWaP1lWNhX6zV6TVyoDUfUb1pvi558txF3l28nakr91K+dACv9G3GwLZ1\nKOGv9zQVhCaL/GgxwNot69dXoUH3fP+lBujdvCavztvC5Ng9mixU8XHuOMR/Cau/sPaLCA6Da1+F\nKwfne6/qtAwbU1fs5b2fd5Ccms7QmHAe7BZBcFDRaOvh7dyaLESkJ/AB4A9MMMa8ke3194Au9odB\nQDVjTLD9tTrABCAUMEBvY8wed8brNBHo8z582s4aYdz1a75v/snsFzVuyS7tF6V838G1VsfXTd9b\nLcHrdYE+71r7RRSgpLtsx3FemrOZHUeT6digCs9f35SG1V23H41yY7IQEX/gE6A7kAisFpHZxpiE\nzGOMMQ9lOf5+4Mosl5gMjDHGLBaRsoDNXbHmS9mq0Oc9+HYILHkbujyd70tpvyjl09IvwpbZVqkp\ncRUElLEmq9veDVUL9vd9b9I5Xp23hcUJR6hTKYhxQ1rTvWl1nf9zA3eOLNoCO40xuwFEZDrQF0i4\nzPEDgRfsxzYFShhjFgMYY5LdGGf+Nb3BKkkteQca9oTarfJ1Ge0XpXzS2SMQPwnivoDkw9by855v\nWF0QAgvW+C45NZ2Pf93JF8v+ooS/8HjPRozoWNdrW2X4Ancmi9rA/iyPE4HonA4UkTCgLvCr/amG\nwCkRmWF//mfgSWNMRrbz7gbuBqhTp45Lg3darzfhr6UwcyTcs8Rqb54Pmf2iFmw6RN+WtV0cpFKF\nKDHOGkVsngm2NGjQDdp+ZP23gDez2myGH9Yk8tbCbRw7m8rNrUJ4vGcjqpfP37875Tx3JoucxoGX\nu138NuD7LMmgBNAJqyy1D/gGGA5MvORixowDxoF1B3fBQ86H0hWh70fw1c3w6yvQY0y+LpPZL+qr\nFXs1WaiiJz3VSg4rx8LBNVCyHLQZYe1AV6WBS95izb6TvDR7M+sTT9MyNJjxQ6NoGRrskmsrx9yZ\nLBKxJqczhQAHL3PsbcCobOeuzVLCmgW0I1uy8BoNulk37MV+Ym3NGNY+z5fQflGqSDpzyCozxX8J\n545B5Qjo9Ta0HAilXDPBnLVFR/XypXhvQCR9I2vj56fzEoUpz8lCRPyAssaYMw4OXQ1EiEhd4ABW\nQvhXy1YRaQRUBGKznVtRRKoaY44B1wBubPzkAt1fgZ2/wKx7YeSfUKpsni9xS1QI7yzaxuTYvbx+\nk/aLUl7KGNi/ClZ+bk1c2zKgYQ9rwrpelwKXmjJpiw7v4tSfuoh8DYwEMoB4oIKIvGuMefty5xhj\n0kVkNLAQa+nsF8aYzSLyMhBnjJltP3QgMN1k6WhojMkQkUeBX8Ra1hAPjM/H5ys8pcpCv8/hy96w\n+DlrpVQeBQeVpG9Lq1/UU721X5TyMjYbbPgGVn4Gh9ZDqQrQ9h5o+x9r8tpFtEWHd3Kq66yIrDPG\ntBSRQUBr4Akg3hjTwt0BOsvtXWedtfAZ687uwT9Y5ak82nTgNH0+WsYL1zfVflHKe5w7bt1TtPNn\nqNrYGkW0GJCvEXRuth62WnQs32W16Hj++qZ6s6qbubrrbICIBAA3Ah8bY9JExDd6m7vaNc/BjsXw\n4/1w3/I83416RW3tF6W8zN5Y+P5OOH8crvs/iBrhkh0js9IWHd7P2f8TY4E9QBlgiX2pq6M5i+Ip\nINAqRyUfgQVP5usSQ2PC2H3sHMt3Jbk4OKXywGaDZe/BpOugRCn4z8/Q5j8uTRRpGTYm/fkXV7/z\nO1+v2sfQmHB+f/RqhsSEa6LwMk793zDGfGiMqW2M6W0se/mnTYfKrnYr6PwobJhu9ZDKo97Na1Kp\nTEkmx+5xeWhKOeVcEkwbAD+/CE2ut+4hqhnp0rdYtuM4vT9YyotzEmheuwILHujEizc0015OXsqp\nZCEiD4hIebFMFJE1WCuU1OV0fgxqtIA5D0LysTydmtkvanHCEQ6euuCmAJW6jH0rYGwna2/r3u/A\nLZMg0HVLufcmneOuyXEMnriS1HQb44a0ZsqIttrLycs5O867075U9lqgKnAH8EbupxRz/gHQbyyk\nnoG5D/KvzX0dGBRdBwNMW7XPPfEplZ3NBsvet1b0+QfAiEXWjpAuKjslp6bzxoKtdH93CX/uPM7j\nPRux+OHOXNushs7NFQHOTnBn/p/sDXxpjFkv+n/XsepN4ZpnrZ31NnwLkQOcPjW0UhBdG2u/KFVI\nzp+wWtbsWAhNboC+Hxe4f1Mmm80wY+0B3vxpq7boKMKc/QkULyKLsJLFQhEph7d1gfVWMaMhNBrm\nPwanD+Tp1MHtwjienMqCTYfcFJxSWDfYfd4Jdv1q3X1962SXJYo1+07S77PlPPrdemoHl2bWqA78\n362RmiiKIGeTxQjgSaCNMeY8UBKrFKUc8fOHGz+zGqrNvj9P5ajOEVUJrxzElNi9bgxQFVvGwPKP\n4Mte1t/TEYsg+m6XlJ2OnEnh4W/WcdOnyzl8+gLvDYhkxr3ttZdTEeZUGcoYYxOREOB2e/XpD2NM\n3pf5FFeV60P3l2H+o1YPnag7nTrNz08Y3C6MV+dtIeHgGZrW0n5RykXOn7D2kd++ABr3gb6fQOmC\n/yDXFh2+y9nVUG8AD2DtRZEA/FdEXndnYD4nagTUuxoWPgsndjt92i2tQwkM8GPKCh1dKBfZvxrG\ndrbuxu75Bgz4qsCJwhjDT5sO0e3dP3h74TY6R1Tl54eu4rEejTVR+Ahny1C9ge7GmC+MMV8APYHr\n3BeWD/Lzs3578/OHWaOs5mtOqBAUwA2RVr+o0xfS3Byk8mnGWJ2Rv+xplZruXAjt7i1w2Wnr4TMM\nmrCSkV+toUzJEnz9n2g+H9Jaezn5mLwsscn6q4drZr+Kmwoh1mZJ+5bDik+dPm1oTDgX0jKYsSbR\njcEpn3bhJEwfBAufhoge1k12Ia0LdMmT5y7y3KxN9P5gKQmHzvBK32bM+29H2msvJ5/k7PjwdWCt\niPyGtYy2M/CU26LyZZEDYctc+OUVaNAdqjV2eMoVtStwZR3tF6XyKTEevhsOZw9Cj9cLPJpIz7Ax\ndeU+3l28neTUdIbGhPNgtwi989rHOdvuYxrW5kMz7F8xxpjp7gzMZ4nA9e9b3TpnjYQM50pLQ9pZ\n/aL+3Kn9opSTjIHYT+GLHoCxyk4x9xUoUfy58zi9P1zKC7M3a4uOYibXZCEirTK/gJpYO9jtB2rZ\nn1P5UbYaXPcuHFwLS9916pTMflFTVuxxb2zKN1w4Cd8MhoVPQUR3e9nJYRfqXM1ae4BBE1aSkqYt\nOoojR2Wo/8vlNYP2h8q/ZjfC1ltgyVvWLmO1WuZ6eGCAPwPahDL2j10cPHWBWsGlCylQVeQcsJed\nzhyEa8dAzKgCT2LH7z3J4z9soF29Sky6oy2BAf6uiVUVGbmOLIwxXXL50kRRUL3egqAqVpuFtBSH\nh9/e1uoX9fVK7RelcmAMrBwLE3tYfZ7uWADtRxc4USSePM89U+KoVSGQzwa11kRRTDl7n8VNOXx1\nFZFq7g7QpwVVsnrwHNsCv7/m8PDMflHTV+8jNd25pbeqmLhwCr4dAgsehwZdYeRSCG1b4Msmp6bz\nn//FkZpuY8KwNlQso3MTxVVe2n1MAAbZv8YDDwN/isgQN8VWPER0h1bD4M8PYd9Kh4cPiQnnePJF\nftp0uBCCU0XCwbUw7irYOh+6vwK3TbN+ESmgDJvhgWlr2XE0mU8HtaJBNdduoaqKFmeThQ1oYoy5\n2RhzM9AUSAWisfbjVgXRYwwEh1p7HF88l+uhnRpU0X5RymIMrBwHE6+1VtXdsQA6/Ne6AdQF3vxp\nK79sPcqL1zelU0RVl1xTFV3O/q0KN8YcyfL4KNDQGHMC0NuKC6pUOavZ4Mk9sPiFXA/N7BcVt/ck\nCQd1Z9tiK+U0fDcMFjxmtZEZuQzqRLvs8t+u3s+4JbsZFhPGkJhwl11XFV3OJoulIjJXRIaJyDBg\nNtZe3GWAU+4LrxgJ7wjt7oPV42HXb7keqv2iirmD62DsVdbNnd1egoHfuKTslGnF7iSembWRThFV\neK5PU5ddVxVtziaLUcCXQEvgSuB/wChjzDljjO7F7Spdn4MqDeHHUdZvjpdRISiAvpG1tV9UcWMM\nrBoPE7tDeioMnwcdH3RZ2QmsLU9HfhVPnUpBfHx7K0r466ZbyuLsHdwGWAb8CvwMLLE/lysR6Ski\n20Rkp4g8mcPr74nIOvvXdhE5le318iJyQEQ+du7jFHEBpeHGz+HsYVjwrz+uSwyJCeNCWgY/xGu/\nqGIh5Qx8f4fV5r5uZ6vsFBbj0rc4k5LGnZNWAzBxWBsqlA5w6fVV0ebs0tlbgVVAf+BWYKWI9Hdw\njj/wCdALa0J8oIhcMqY1xjxkjGlpjGkJfITVSiSrV4A/nInRZ4S0hk4Pw/qvYeu8yx6W2S/qqxV7\ncSJvq6Ls0AZrtVPCbOj6Atz+HZSp7NK3SM+wMWrqGvYmnefzwa0Jr1LGpddXRZ+zY8xnsHbJG2aM\nGQq0BZ5zcE5bYKcxZrcx5iIwHeiby/EDgWmZD0SkNVAdWORkjL6j8+NQoznMeQDOXb4X1NCYMHYf\n135RPssYiPsCJnSDtAswfK71i4QLy06ZXp23haU7jjOm3xW0q+faRKR8g7N/6/yMMUezPE5y4tza\nWH2kMiXan/sXEQkD6mKVuRARP6xWI4/l9gYicreIxIlI3LFjxxyEU4SUKAn9xlo3Ws176LJbsfa6\nwuoXNTl2T6GGpwpB6ln4YQTMfcha/DByGYS1d8tbTVmxl0nL93BXp7oMaFPHLe+hij5nk8VPIrJQ\nRIaLyHBgHjDfwTk59Ri4XL3buXCEAAAbVklEQVTkNuB7Y0zmbcn3AfONMfsvc7x1MWPGGWOijDFR\nVav62Drw6s2gy9OQ8CNs+iHHQzL7Rf285QgHTl0o5ACV2xzeaK122jwTrnkOBn0PZdyzR8SyHcd5\ncfZmujauxpO9mrjlPZRvcHaC+zFgHNACiATGGWMc3YyXCIRmeRwCHLzMsbeRpQQFxACjRWQP8A4w\n1L61a/HS4QEIaQPzHoEzh3I8ZFC01S9qmvaLKvqMgbgvYXxX6+bMYXOg86NuKTsB7DqWzH1T42lQ\ntSwfDLwSfz/dJ0VdntN/C40xPxhjHrZPSs904pTVQISI1BWRklgJYXb2g0SkEVARiM3yXoOMMXWM\nMeHAo8BkY0zuy4N8kZ+/VY5KT4XZ9+dYjgqpqP2ifELqWZhxF8x90Co3jVxmlZ/c5NT5i4yYtJoA\nfz8mDIuirO6TrRxwtJ/FWRE5k8PXWRHJ9fZhY0w6MBpYCGwBvjXGbBaRl0XkhiyHDgSmO7MUt1iq\nXB+6vww7F8OayTkeov2iirjDm2Dc1Va5scuzMHgGlHVfWTUtw8a9X63h4KkUxg1tTWgl3StbOSa+\n8jM6KirKxMXFeToM97DZYEpfOLAG7v0TKoZne9lwzf/9TpWypfj+XvdMgio3MMb6BWDB4xBYAW6e\nCHU7ufktDU/P3Mi0Vft5b0Ak/a4Mcev7Ke8nIvHGGIc7Y+ntmUWBnx/0/RQQmDXKSh6XvKz9ooqc\n1GSrceSc/0JotFV2cnOiAPjizz1MW7WfUV3qa6JQeaLJoqgIDoVeb8DeZbDy83+9/E+/qD2FH5vK\nmyMJML4LbPgWrn4ahsy0ttp1s9+2HmXMvAR6NqvBI90buf39lG/RZFGUtBwEDXvBLy/Bse2XvPRP\nv6iDJCWneihAlStjYM0UGH+NdQ/N0B/h6ieshQxutu3wWe6ftpYmNcvz7oBI/HTlk8ojTRZFiQhc\n/4HVQ2rWSMhIv+TluzrXJcNmeH72Zg8FqC7r4jmYdS/MHg0hUVbZqd5VhfLWx5NTGfG/1QSV9GfC\nsCiCSurKJ5V3miyKmnLV4bp34UA8/PneJS81qFaOB7pFMG/DIeZvzPm+DOUBRxJgXBdYPx2uetIa\nUZSrXihvnZqewcgp8Rw7m8r4oVHUrFC6UN5X+R5NFkXRFTfBFTfD729aTeayuKdzPVqEVOC5WZu0\nHOVpx7bBjHvg845w4YQ1N9HlqUIpO4G18umpGRuJ23uS/7s1ksjQ4EJ5X+WbNFkUVb3fsTa8mTnS\numnProS/H+/cEsnZlHSe/1HLUR5xcB18MwQ+iYYtsyF6JNy7HOoX7tYvn/+xmxlrDvBQt4b0aVGr\nUN9b+R5NFkVVUCW44SM4uhl+v7QTSsPq9nLUxkPM26DlqEKzbwV81d9qJ777d+j0CDy4EXq+Viir\nnbJauPkwby3cyvWRtfhv1waF+t7KN+lMV1HWsAdcOQT+fB8a9YLQtn+/dE/neizafJjnftxEdL1K\nVClbyoOB+jBjYPdvsOT/rGXNQZWt5n9t77JutPOATQdO8+D0dbQICebt/i0Q0ZVPquB0ZFHU9XgN\nyodY5aiL5/9+OrMclZySzvM/bvJggD7KZrM2pxp/DUzpByd2QY/XrZFE50c9liiOnknhrslxBAcF\nMH5IawIDCmd+RPk+TRZFXWB5uPET64fVzy9e8lJE9XI82D2C+RsPM3fD5Rr+qjyxZcDG7+HzDjD9\ndjifBH3ehwfWQ8x9UNJzO8ylpGVw15R4Tp1PY8KwKKqVD/RYLMr3aLLwBXU7Q/S9sGos7L50F9q7\nO9UjMjSY53/czHFdHZV/6RetPk4fR1mbEtkyoN84uH8NRN0BJTxb5jPG8Nj3G9iQeIr3b2tJs1qe\nGdko36XJwld0fR4qN7Bu/FozBc6fAOzlqP4tSE5J57lZm3S/7rxKuwArx8KHV1pt4kuVg1unwH0r\nIHIA+HvHtN+Hv+xkzvqDPN6jMT2a1fB0OMoHabLwFSWD4OYJ4B9g3SX8TgRMuQnWTCGiXBoPdW/I\ngk2Hmauro5yTcgaWvQfvN7e6wlYIsXasu/sPaHqD2zYkyo+5Gw7y3s/bualVbUZeVc/T4SgfpS3K\nfY0xcGgdbJ5lbct5ai/4lcBW9yo+OXIFP5yL5LuH+1C1nK6OytH5E1ajxpWfQ8ppqH8NdHoUwjt4\nOrIcrd9/ilvHxtK8dgWm3hVNqRI6oa3yxtkW5ZosfFkOiSPN+LM9qBVNuw1Bmlxv3a+h4OwRiP0I\nVn8Baeeg0XXQ+RGo3drTkV3WodMX6Pvxn5Qs4cePozpQWZdHq3zQZKEuZU8caxZMovLe+YT5HQW/\nElD3Kmh2IzTuUzwTx6l98OcH1jyPLQ2a3QSdHobqzTwdWa7OX0znls9j2Zt0nh/ubU+jGuU8HZIq\nojRZqBylZ9i4+bPlBCVtYmJUIkE758DJPfbE0Rma9SseieP4Tlj2Lmz4BhCIvA06PmRtY+vlbDbD\nfVPXsCjhMBOHtaFL48K9O1z5Fk0W6rJ2Hj1L7w+X0aVRVT4f1Ao5vAES7KWqk3tA/K322U1vBF8r\nVR3eCEv/zyrNlSgFrYZBh/9aE9hFxFs/beXT33fxXJ+mjOhY19PhqCJOk4XK1ed/7OKNBVv54LaW\n9G1Z23rSGDi03p44ZsHJv3wncexfDUvfge0/Qcly0GYExIwq9J5NBTVjTSIPf7uegW1Dea1fc23l\noQpMk4XKVYbNcPNny9mTdI5FD3WmWrlsd/saA4c3WKONrIkja6mqTGXPBO8sY2DPUljyNvy1BEpX\ntG5ejL7b+r6IidtzgtvHr6RVWDBTRkQT4O89y3dV0aXJQjm082gyvT9cytUNqzJ2SOvL/5aaa+K4\nERpf712JwxjYsQiWvAOJq6BsdYgZDVF3Qqmyno4uX/afOM+Nn/xJucASzBrVgeCgkp4OSfkITRbK\nKWP/2MXr2ctRufk7cWTOcXhR4rBlQMKPsPRdOLIRKoRChweszrwBRbdP0tmUNPp/FsvB0xeYNaoD\n9asWzYSnvJNXJAsR6Ql8APgDE4wxb2R7/T0gc0eYIKCaMSZYRFoCnwHlgQxgjDHmm9zeS5NF/mTY\nDP0/X85fxy9TjspN1sSRMAtO7PZM4shIg43fWUkiaYfV9qTjw9DiVuuO9iIsw2a4a3Icf2w/xv/u\naEvHiCqeDkn5GI8nCxHxB7YD3YFEYDUw0BiTcJnj7weuNMbcKSINAWOM2SEitYB4oIkx5tTl3k+T\nRf5llqOualiVcbmVo3JjjLXSaPPMbImjk32Oww2JIy0F1n1l3Sdxah9Ub27dI9G0b6FtXepur85N\nYMKyv3jlxisY0i7M0+EoH+RssnBnF7S2wE5jzG57QNOBvkCOyQIYCLwAYIzZnvmkMeagiBwFqgKX\nTRYq/xpUK8uj1zbktflbmb3+oHPlqOxEoGYL66vr81biyFyOO+cBmPuwlTgyV1WVKcBvyKnJEP8l\nLP8Ykg9DSBvo9ba1GZQPrQ6avmofE5b9xbCYME0UyuPcmSxqA/uzPE4EonM6UETCgLrArzm81hYo\nCexyQ4zKbkTHevy06TAvzN5MTP3KeStHZZc1cVzz3KWJY+6DMO+R/CWOCydh1XhY8an1fd3OcNM4\n678+lCQAYncl8eysTXSKqMJzfZp6Ohyl3FqGugXoYYz5j/3xEKCtMeb+HI59AgjJ/pqI1AR+B4YZ\nY1bkcN7dwN0AderUab13716Xf47iZNexZHp/sJTOBSlH5SazVJV5H8eJXVapKryjVaq6XOJIPgYr\nPoFVE+DiWWjY02ruF9rGtfF5iT3Hz3Hjp39SuUxJZtzXgQqli/a8i/Ju3jBnEQO8aIzpYX/8FIAx\n5vUcjl0LjDLGLM/yXHmsRPG6MeY7R++ncxauMX7JbsbM38L7A1py45X5KEc5yxg4sumf5bg5JY70\nVFj+IcT/D9JTrEnzTo9Ajebui8vDTl9I46ZP/yTp3EV+HNWBsMqe23lPFQ/ekCxKYE1wdwUOYE1w\n326M2ZztuEbAQqCusQcjIiWBBcAcY8z7zryfJgvXyLAZbh0by86jySx+qHPhbM35d+Kwl6oyE4f4\nAQZaDLD6NlWJcH8sHpSeYeOOSatZsTuJKSOiaVfPi+5dUT7L4xPcxph0ERmNlQj8gS+MMZtF5GUg\nzhgz237oQGC6uTRr3Qp0BiqLyHD7c8ONMevcFa+y+PsJb/dvQa8PlvL0zE2MH+qGclR2ItZooUZz\nuObZfxJHRiq0uQsqFo/J3ZfnJrB0x3HevLm5JgrldfSmPJWjCUt38+q8Lbw3IJJ+VxadJntF1eTY\nPTz/42bu6lSXZ67TCW1VeJwdWWhzGZWjOzrUJSqsIi/OTuDomRRPh+PTlu44xktzEujauBpP9mri\n6XCUypEmC5Ujfz/hrf4tSEnL4OmZG/GVEai32Xk0mfumriGiWlk+GHgl/n6+tQRY+Q5NFuqy6lUt\ny2M9GvHzlqPMXHvA0+H4nJPnLjLif6spVcKPCcOiKFvKnbc9KVUwmixUrv4pR23miJajXOZiuo2R\nX8Vz6FQKY4e0JqRikKdDUipXmixUrvz9hLdvieRiho2nZ2g5yhWMMTw3axMr/zrBm/2b0zqsiG4o\npYoVTRbKobpVyvBYj8b8svUoM9ZoOaqgJi77i2/i9jOqS31daaaKDE0Wyil3tA+nTXhFXpqj5aiC\n+GXLEcbM30LPZjV4pHsjT4ejlNM0WSin+PkJb/W3ylFPaTkqX7YePsN/p62lWa3yvDsgEj9d+aSK\nEE0WymmZ5ahftx7lBy1H5cnx5FRGTIqjTKkSjB8aRVBJXfmkihZNFipPspajDp/WcpQzUtIyuGdK\nPMeTUxk/NIqaFUp7OiSl8kyThcoTPz/h7f6RpGXYeGrGBi1HOXDhYgaPfLee+L0n+b9bI4kMDfZ0\nSErliyYLlWfhVcrweI/G/LbtGN/HJ3o6HK+16cBp+ny0lHkbDvFkr8b0aVHL0yEplW+aLFS+DG8f\nTtvwSrw8N0HLUdlk2Ayf/b6Lfp/+SXJqOl+NiGbkVfU9HZZSBaLJQuWLn713VFqGjSe1HPW3A6cu\ncPv4Fbz501a6NanOTw90pmNEAfYbV8pLaLJQ+RZepQxP9GzM79uO8Z2Wo5i9/iA931/CpgOnebt/\nCz4d1IqKZUp6OiylXELX76kCGRYTzoJNh3llTgKdIqoUy5U+Z1LSeH7WJmatO0irOsG8N6Clboeq\nfI6OLFSB+Nl31ku3GZ78ofjdrLfqrxP0en8pczYc4sFuEXx7T4wmCuWTNFmoAgurXIYnejbij+3H\n+C6ueJSj0jJsvL1wK7eNi6WEv/DdyBge7NaQEv76T0r5Ji1DKZcYmlmOmptAx4gq1Ar23XLUrmPJ\nPPTNOjYknubWqBCev76Z7kWhfJ7+GqRcIvNmvXSb8dneUcYYpq7cS58Pl7HvxHk+H9yKt/pHaqJQ\nxYImC+UydSoH8WSvxj5ZjkpKTuWuyfE8M3MTUeEVWfhgZ3peUdPTYSlVaPRXIuVSQ9qFsWDTIZ8q\nR/227SiPfbeBMylpPNenKXe0D9eOsarY0ZGFcik/P+GtmyPJMIYni3g5KiUtgxd+3MQdX66mStmS\nzB7dgREd62qiUMWSW5OFiPQUkW0islNEnszh9fdEZJ39a7uInMry2jAR2WH/GubOOJVrZZajlmw/\nxrdx+z0dTr5sPniaPh8t43+xexnRsS6zRnWgcY3yng5LKY9xWxlKRPyBT4DuQCKwWkRmG2MSMo8x\nxjyU5fj7gSvt31cCXgCiAAPE28896a54lWsNjg5j/sZDvDp3C50iqhaZcpTNZhi/dDfvLNpGpTIl\nmTKiLZ0iqno6LKU8zp0ji7bATmPMbmPMRWA60DeX4wcC0+zf9wAWG2NO2BPEYqCnG2NVLpa5Oqoo\nlaMOnrrA7RNW8PqCrXRtbPV10kShlMWdyaI2kLUGkWh/7l9EJAyoC/ya13OV9wqtFMRT9nLUN6u9\nuxw1x97XaWPiad7q34LPBmtfJ6WycmeyyGkW8HK/Xt4GfG+MycjLuSJyt4jEiUjcsWPH8hmmcqdB\n0WHE1KvMq/O2cODUBU+H8y9nU9J4+Jt13D9tLfWrlWX+A524NSoUEZ3EViordyaLRCA0y+MQ4OBl\njr2Nf0pQTp9rjBlnjIkyxkRVrarlAm+U2crcZgxP/uBdrcxX7zlBrw+W8uP6gzzYLYLvtK+TUpfl\nzmSxGogQkboiUhIrIczOfpCINAIqArFZnl4IXCsiFUWkInCt/TlVBIVWCuKp3k1YuuM4072gHJWW\nYeOdhdsYMDYWPxG+vUf7OinliNtWQxlj0kVkNNYPeX/gC2PMZhF5GYgzxmQmjoHAdJPlV05jzAkR\neQUr4QC8bIw54a5YlfsNaluH+RsOMWbeFjo3rEptD62O2m3v67Q+8TS3tA7hhRu0r5NSzhBvKgsU\nRFRUlImLi/N0GCoX+0+cp8f7S2gdVpHJd7Yt1HkBYwzTV+/n5TkJlCzhxxs3NadXc23XoZSIxBtj\nohwdp+NuVWiylqOmrSq8clRScip3T4nnqRkbaR1m9XXSRKFU3uj4WxWqQW3rsGDjIcbMS6BzwyqE\nVAxy6/v9vu0oj32/gdPn03j2uibc2UHbdSiVHzqyUIXKz0948+YWAG7dWS8lLYMXZ29m+JerqRRU\nkh9Hd+A/neppolAqnzRZqEKXWY5atvM4X6/a5/Lrbz54mus/Wsak5Xu4o0M4P47uQJOa2tdJqYLQ\nMpTyiEHRdViw6RCvzdtC54iqhFYqeDnKZjNMWLabdxZuJzgogMl3tqVzQ73/RilX0JGF8giRLOWo\nGQW/We/Q6QsMnriS1+ZvpUvjqvz0YGdNFEq5kCYL5TEhFYN4+rom/Lkziakr81+OmrfhED3fX8q6\n/ad46+YWfD64NZW0r5NSLqVlKOVRt7etw4KNh3l9/hauapi3ctTZlDRemL2ZGWsO0DI0mPcHtCS8\nirbrUModdGShPEpEeOPm5ogIT/ywAZvNuXJU3J4T9P5wKbPWHuC/XSP4bmSMJgql3EiThfK4kIpB\nPN27Cct3JTHVweqotAwb7y7axq1jYxGE70a25+HuDQnQvk5KuZWWoZRXGNg2lAWbDvH6/C1cfZly\n1F/Hz/HgN+tYv/8U/VuH8KL2dVKq0OivY8orWOWoFviJ8Pj3l5ajjDFMX7WP6z5cyp7j5/h0UCve\nuSVSE4VShUiThfIatYNL88x1TYjdncTUlXsBOHHuIvdMiefJGRu5sk4wPz3Yid7a10mpQqe/mimv\nclubUOZvPMTrC7ZSqoQ/by/apn2dlPICOrJQXuWSctQPG6gYFMCsUdrXSSlP05GF8jq1g0vz0cAr\n2ZB4mnuuqkdggL+nQ1Kq2NNkobxSl8bV6NK4mqfDUErZaRlKKaWUQ5oslFJKOaTJQimllEOaLJRS\nSjmkyUIppZRDmiyUUko5pMlCKaWUQ5oslFJKOSQF3fvYW4jIMWBvAS5RBTjuonA8yVc+B+hn8Va+\n8ll85XNAwT5LmDHG4Yb1PpMsCkpE4owxUZ6Oo6B85XOAfhZv5SufxVc+BxTOZ9EylFJKKYc0WSil\nlHJIk8U/xnk6ABfxlc8B+lm8la98Fl/5HFAIn0XnLJRSSjmkIwullFIOabKwE5FXRGSDiKwTkUUi\nUsvTMeWXiLwtIlvtn2emiAR7Oqb8EpFbRGSziNhEpMitXBGRniKyTUR2isiTno6nIETkCxE5KiKb\nPB1LQYhIqIj8JiJb7H+3HvB0TPklIoEiskpE1ts/y0tuey8tQ1lEpLwx5oz9+/8CTY0xIz0cVr6I\nyLXAr8aYdBF5E8AY84SHw8oXEWkC2ICxwKPGmDgPh+Q0EfEHtgPdgURgNTDQGJPg0cDySUQ6A8nA\nZGPMFZ6OJ79EpCZQ0xizRkTKAfHAjUXx/4uICFDGGJMsIgHAMuABY8wKV7+XjizsMhOFXRmgyGZR\nY8wiY0y6/eEKIMST8RSEMWaLMWabp+PIp7bATmPMbmPMRWA60NfDMeWbMWYJcMLTcRSUMeaQMWaN\n/fuzwBagtmejyh9jSbY/DLB/ueVnlyaLLERkjIjsBwYBz3s6Hhe5E1jg6SCKqdrA/iyPEymiP5R8\nlYiEA1cCKz0bSf6JiL+IrAOOAouNMW75LMUqWYjIzyKyKYevvgDGmGeMMaHAVGC0Z6PNnaPPYj/m\nGSAd6/N4LWc+SxElOTxXZEesvkZEygI/AA9mqywUKcaYDGNMS6wKQlsRcUuJsIQ7LuqtjDHdnDz0\na2Ae8IIbwykQR59FRIYBfYCuxssnpvLw/6WoSQRCszwOAQ56KBaVhb2+/wMw1Rgzw9PxuIIx5pSI\n/A70BFy+CKFYjSxyIyIRWR7eAGz1VCwFJSI9gSeAG4wx5z0dTzG2GogQkboiUhK4DZjt4ZiKPfuk\n8ERgizHmXU/HUxAiUjVztaOIlAa64aafXboayk5EfgAaYa282QuMNMYc8GxU+SMiO4FSQJL9qRVF\neGVXP+AjoCpwClhnjOnh2aicJyK9gfcBf+ALY8wYD4eUbyIyDbgaq8PpEeAFY8xEjwaVDyLSEVgK\nbMT69w7wtDFmvueiyh8RaQH8D+vvlx/wrTHmZbe8lyYLpZRSjmgZSimllEOaLJRSSjmkyUIppZRD\nmiyUUko5pMlCKaWUQ5oslMoDEUl2fFSu538vIvXs35cVkbEissveMXSJiESLSEn798Xqplnl3TRZ\nKFVIRKQZ4G+M2W1/agJWY74IY0wzYDhQxd508BdggEcCVSoHmiyUygexvG3vYbVRRAbYn/cTkU/t\nI4W5IjJfRPrbTxsE/Gg/rj4QDTxrjLEB2LvTzrMfO8t+vFJeQYe5SuXPTUBLIBLrjubVIrIE6ACE\nA82Baljtr7+wn9MBmGb/vhnW3egZl7n+JqCNWyJXKh90ZKFU/nQEptk7fh4B/sD64d4R+M4YYzPG\nHAZ+y3JOTeCYMxe3J5GL9s15lPI4TRZK5U9O7cdzex7gAhBo/34zECkiuf0bLAWk5CM2pVxOk4VS\n+bMEGGDfeKYq0BlYhbWt5c32uYvqWI33Mm0BGgAYY3YBccBL9i6oiEhE5h4eIlIZOGaMSSusD6RU\nbjRZKJU/M4ENwHrgV+Bxe9npB6x9LDZh7Ru+EjhtP2celyaP/wA1gJ0ishEYzz/7XXQBilwXVOW7\ntOusUi4mImWNMcn20cEqoIMx5rB9v4Hf7I8vN7GdeY0ZwFNFeP9x5WN0NZRSrjfXviFNSeAV+4gD\nY8wFEXkBax/ufZc72b5R0ixNFMqb6MhCKaWUQzpnoZRSyiFNFkoppRzSZKGUUsohTRZKKaUc0mSh\nlFLKIU0WSimlHPp/m/Bj8kiKDB0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xc1bf550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "logloss_s1 =np.array(logloss_s).reshape(len(Cs),len(penaltys))\n",
    "x_axis = np.log10(Cs)\n",
    "for j, onePenalty in enumerate(penaltys):\n",
    "    pyplot.plot(x_axis, np.array(logloss_s1[:,j]), label = ' Test-' + onePenalty)\n",
    "\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'logloss' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用最佳参数在训练集上训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bestCs = logloss_s1.argmin(axis = 0)\n",
    "\n",
    "best_logloss = logloss_s1[bestCs[0],0]\n",
    "best_penalty_index = 0\n",
    "best_penalty = penaltys[best_penalty_index]\n",
    "\n",
    "for j, onePenalty in enumerate(penaltys):\n",
    "    if logloss_s1[bestCs[j],j] < best_logloss:\n",
    "        best_logloss = logloss_s1[bestCs[j],j]\n",
    "        best_penalty_index = j\n",
    "        best_penalty = penaltys[best_penalty_index]\n",
    "       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best C: 0.100000 \n",
      " best penalty: l1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l1', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bestCs = logloss_s1.argmin(axis = 0)\n",
    "\n",
    "best_logloss = logloss_s1[bestCs[0],0]\n",
    "best_penalty_index = 0\n",
    "best_penalty = penaltys[best_penalty_index]\n",
    "\n",
    "for j, onePenalty in enumerate(penaltys):\n",
    "    if logloss_s1[bestCs[j],j] < best_logloss:\n",
    "        best_logloss = logloss_s1[bestCs[j],j]\n",
    "        best_penalty_index = j\n",
    "        best_penalty = penaltys[best_penalty_index]\n",
    "\n",
    "bestC = Cs[bestCs[best_penalty_index]]\n",
    "\n",
    "print(\"best C: %f \\n best penalty: %s\"%(bestC, best_penalty) )\n",
    "    \n",
    "LR = LogisticRegression(penalty=best_penalty, C=bestC)\n",
    "LR.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成提交结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    }
   ],
   "source": [
    "y_test_pred = LR.predict_proba(X_test)\n",
    "out_df1 = pd.DataFrame(y_test_pred)\n",
    "out_df1.columns = [\"high\", \"medium\", \"low\"]\n",
    "\n",
    "out_df = pd.concat([test_Id,out_df1], axis = 1)\n",
    "out_df.to_csv(\"./data/LR_Rent.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
